{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import checklist\n",
    "import spacy\n",
    "import itertools\n",
    "\n",
    "import checklist.editor\n",
    "import checklist.text_generation\n",
    "from checklist.test_types import MFT, INV, DIR\n",
    "from checklist.expect import Expect\n",
    "import numpy as np\n",
    "import spacy\n",
    "from checklist.test_suite import TestSuite\n",
    "from checklist.perturb import Perturb\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<checklist.text_generation.TextGenerator at 0x7fb194f93da0>"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "editor = checklist.editor.Editor()\n",
    "editor.tg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# editor.template((('{first_name} is a good guy', 'Who is a good guy?'), '{first_name}', '3') )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "r = csv.DictReader(open('/home/marcotcr/datasets/airline/Tweets.csv'))\n",
    "labels = []\n",
    "confs = []\n",
    "airlines = []\n",
    "tdata = []\n",
    "reasons = []\n",
    "for row in r:\n",
    "    sentiment, conf, airline, text = row['airline_sentiment'], row['airline_sentiment_confidence'], row['airline'], row['text']\n",
    "    labels.append(sentiment)\n",
    "    confs.append(conf)\n",
    "    airlines.append(airline)\n",
    "    tdata.append(text)\n",
    "    reasons.append(row['negativereason'])\n",
    "\n",
    "mapping = {'negative': 0, 'positive': 2, 'neutral': 1}\n",
    "labels = np.array([mapping[x] for x in labels]).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp = spacy.load('en_core_web_sm')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentences = tdata\n",
    "parsed_data = list(nlp.pipe(sentences))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "suite = TestSuite()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Capability: Vocabulary"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### MFTs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "air_noun = ['flight', 'seat', 'pilot', 'staff', 'service', 'customer service', 'aircraft', 'plane', 'food', 'cabin crew', 'company', 'airline', 'crew']\n",
    "editor.add_lexicon('air_noun', air_noun)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "great, good, excellent, amazing, bad, terrible, incredible, awful, expensive, extraordinary, new, interesting, wonderful, nice, fantastic, real, important, awesome, unusual, American, different, poor, exceptional, beautiful, ordinary, little, impressive, special, enormous, actual, easy, big, horrible, huge, fine, lousy, perfect, international, decent, terrific\n"
     ]
    }
   ],
   "source": [
    "print(', '.join(editor.suggest('It was {a:mask} {air_noun}.')[:40]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "pos_adj = ['good', 'great', 'excellent', 'amazing', 'extraordinary', 'beautiful', 'fantastic', 'nice', 'incredible', 'exceptional', 'awesome', 'perfect', 'fun', 'happy', 'adorable', 'brilliant', 'exciting', 'sweet', 'wonderful']\n",
    "neg_adj = ['awful', 'bad', 'horrible', 'weird', 'rough', 'lousy', 'unhappy', 'average', 'difficult', 'poor', 'sad', 'frustrating', 'hard', 'lame', 'nasty', 'annoying', 'boring', 'creepy', 'dreadful', 'ridiculous', 'terrible', 'ugly', 'unpleasant']\n",
    "neutral_adj = ['American', 'international',  'commercial', 'British', 'private', 'Italian', 'Indian', 'Australian', 'Israeli', ]\n",
    "editor.add_lexicon('pos_adj', pos_adj, overwrite=True)\n",
    "editor.add_lexicon('neg_adj', neg_adj, overwrite=True )\n",
    "editor.add_lexicon('neutral_adj', neutral_adj, overwrite=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "liked, enjoyed, like, appreciate, appreciated, enjoy, loved, love, miss, missed, hate, needed, likes, wanted, got, recommend, prefer, admired, value, need, respected, want, dislike, enjoying, respect, enjoys, admire, was, feel, liking, dig, hated, mean, preferred, underestimated, disliked, used, get, use, valued, understand, did, found, adore, helped, dug, trust, remember, noticed, about, tried, hit, had, regret, took, felt, praised, cherish, have, left, loves, understood, do, LOVE, compliment, trusted, bought, thank, treasure, applaud, support, rate, know, commend, credit, underestimate, see, supported, think, thought, all, saw, picked, believe, beat, welcome, considered, impressed, improved, met, chose, thanks, deserved, envy, blame, for, help, packed, recommended, in, follow, choose, loving, misses, is, into, lost, consider, changed, worked, meant, leave, joined, wish, regretted, made, earned, rocked, cherished, take, worth, finished, experienced, are, fancy, handled, thanked, followed, saved, spoiled, told, surprised, salute, meet, pleased, try, recruited, crave, owe, reviewed, doubt, values, forgive, hired, hope, remembered, stressed, praise, respects, just, LIKE, knew, am, welcomed, tested, sold, Love, Like, survived, ate, planned, drove, ordered, sampled, crashed, managed, brought, ended, deserve, rode, eat, applauded, regrets, spent, heard, grabbed, needs, tasted, coveted, ruined, timed, delayed, caught, dreaded, watched, forgot, anticipated, canceled, own, rushed, cooked, taste, delivered, prefers, filled, deserves, shortened, mind, received, edited\n"
     ]
    }
   ],
   "source": [
    "print(', '.join(editor.suggest('I really {mask} the {air_noun}.')[:200]))\n",
    "# print()\n",
    "# print(', '.join(editor.suggest('I {mask} the {air_noun}.')[:50]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "pos_verb_present = ['like', 'enjoy', 'appreciate', 'love',  'recommend', 'admire', 'value', 'welcome']\n",
    "neg_verb_present = ['hate', 'dislike', 'regret',  'abhor', 'dread', 'despise' ]\n",
    "neutral_verb_present = ['see', 'find']\n",
    "pos_verb_past = ['liked', 'enjoyed', 'appreciated', 'loved', 'admired', 'valued', 'welcomed']\n",
    "neg_verb_past = ['hated', 'disliked', 'regretted',  'abhorred', 'dreaded', 'despised']\n",
    "neutral_verb_past = ['saw', 'found']\n",
    "editor.add_lexicon('pos_verb_present', pos_verb_present, overwrite=True)\n",
    "editor.add_lexicon('neg_verb_present', neg_verb_present, overwrite=True)\n",
    "editor.add_lexicon('neutral_verb_present', neutral_verb_present, overwrite=True)\n",
    "editor.add_lexicon('pos_verb_past', pos_verb_past, overwrite=True)\n",
    "editor.add_lexicon('neg_verb_past', neg_verb_past, overwrite=True)\n",
    "editor.add_lexicon('neutral_verb_past', neutral_verb_past, overwrite=True)\n",
    "editor.add_lexicon('pos_verb', pos_verb_present+ pos_verb_past, overwrite=True)\n",
    "editor.add_lexicon('neg_verb', neg_verb_present + neg_verb_past, overwrite=True)\n",
    "editor.add_lexicon('neutral_verb', neutral_verb_present + neutral_verb_past, overwrite=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Individual words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "test = MFT(pos_adj + pos_verb_present + pos_verb_past, labels=2)\n",
    "suite.add(test, 'single positive words', 'Vocabulary', '')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "test = MFT(neg_adj + neg_verb_present + neg_verb_past, labels=0)\n",
    "suite.add(test, 'single negative words', 'Vocabulary', '')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "test = MFT(neutral_adj + neutral_verb_present + neutral_verb_past, labels=1)\n",
    "suite.add(test, 'single neutral words', 'Vocabulary', 'TODO_DESCRIPTION')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Words in context"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = editor.template('{it} {air_noun} {be} {pos_adj}.', it=['The', 'This', 'That'], be=['is', 'was'], labels=2, save=True)\n",
    "t += editor.template('{it} {be} {a:pos_adj} {air_noun}.', it=['It', 'This', 'That'], be=['is', 'was'], labels=2, save=True)\n",
    "t += editor.template('{i} {pos_verb} {the} {air_noun}.', i=['I', 'We'], the=['this', 'that', 'the'], labels=2, save=True)\n",
    "t += editor.template('{it} {air_noun} {be} {neg_adj}.', it=['That', 'This', 'The'], be=['is', 'was'], labels=0, save=True)\n",
    "t += editor.template('{it} {be} {a:neg_adj} {air_noun}.', it=['It', 'This', 'That'], be=['is', 'was'], labels=0, save=True)\n",
    "t += editor.template('{i} {neg_verb} {the} {air_noun}.', i=['I', 'We'], the=['this', 'that', 'the'], labels=0, save=True)\n",
    "# equivalent to:\n",
    "# test = MFT(t.data, labels=t.labels, templates=t.templates)\n",
    "test = MFT(**t)\n",
    "suite.add(test, 'Sentiment-laden words in context', 'Vocabulary', 'Use positive and negative verbs and adjectives with airline nouns such as seats, pilot, flight, etc. E.g. \"This was a bad flight\"')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['see', 'find', 'saw', 'found']"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "editor.lexicons['neutral_verb']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = editor.template('{it} {air_noun} {be} {neutral_adj}.', it=['That', 'This', 'The'], be=['is', 'was'], save=True)\n",
    "t += editor.template('{it} {be} {a:neutral_adj} {air_noun}.', it=['It', 'This', 'That'], be=['is', 'was'], save=True)\n",
    "t += editor.template('{i} {neutral_verb} {the} {air_noun}.', i=['I', 'We'], the=['this', 'that', 'the'], save=True)\n",
    "test = MFT(t.data, labels=1, templates=t.templates)\n",
    "suite.add(test, 'neutral words in context', 'Vocabulary', 'Use neutral verbs and adjectives with airline nouns such as seats, pilot, flight, etc. E.g. \"The pilot is American\"')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Intensifiers and reducers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "absolutely , really , very , truly , pretty , just , simply , quite , most , incredibly , totally , amazingly , extremely , absolute , almost , rather , completely , super , utterly , especially , unbelievably , equally , surprisingly , exceptionally , extraordinarily , particularly , overall , entirely , genuinely , otherwise , insanely , obviously , fucking , fairly , altogether , amazing , actually , undeniably , all , unusually , damn , incredible , exceedingly , awesome , already , entire , an , oddly , extra , actual\n"
     ]
    }
   ],
   "source": [
    "print(' , '.join(editor.suggest('{it} {be} {a:mask} {pos_adj} {air_noun}.', it=['It', 'This', 'That'], be=['is', 'was'])[:50]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "intens_adj = ['very', 'really', 'absolutely', 'truly', 'extremely', 'quite', 'incredibly', 'amazingly', 'especially', 'exceptionally', 'unbelievably', 'utterly', 'exceedingly', 'rather', 'totally', 'particularly']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "really, always, also, greatly, truly, certainly, I, thoroughly, especially, still, personally, particularly, highly, all, definitely, we, sincerely, very, much, most, so, deeply, fully, both, absolutely, just, and, people, strongly, simply, obviously, do, actually, have, clearly, quite, sure, rather, too, you, generally, they, would, did, genuinely, will, again, completely, totally, honestly, never, seriously, should, even, guys, only, often, can, must, to, now, who, immediately, many, extremely, then, does, mostly, immensely, friends, dearly, everyone, kindly, usually, shall, ever, may, respectfully, tremendously, already, hugely, students, probably, had, please, least, desperately, incredibly, that, long, REALLY, enthusiastically, could, he, secretly, therefore, readers, me, family, further\n"
     ]
    }
   ],
   "source": [
    "print(', '.join(editor.suggest('{i} {mask} {pos_verb} {the} {air_noun}.', i=['I', 'We'], the=['this', 'that', 'the'])[:100]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "30e4f8680fd14265987ab5b89ecb0b76",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "TemplateEditor(bert_suggests=['enjoyed', 'liked', 'loved', 'appreciated', 'missed', 'enjoy', 'needed', 'apprec…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "editor.visual_suggest('I really {mask} the flight.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "intens_verb = [ 'really', 'absolutely', 'truly', 'extremely',  'especially',  'utterly',  'totally', 'particularly', 'highly', 'definitely', 'certainly', 'genuinely', 'honestly', 'strongly', 'sure', 'sincerely']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "monotonic_label = Expect.monotonic(increasing=True, tolerance=0.1)\n",
    "non_neutral_pred = lambda pred, *args, **kwargs: pred != 1\n",
    "monotonic_label = Expect.slice_pairwise(monotonic_label, non_neutral_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['That was an incredible plane.',\n",
       "  'That was an exceptionally incredible plane.'],\n",
       " ['That was an exciting plane.', 'That was an amazingly exciting plane.'],\n",
       " ['It was a beautiful service.', 'It was a particularly beautiful service.'],\n",
       " ['It was an exceptional seat.', 'It was a rather exceptional seat.'],\n",
       " ['It was a happy customer service.',\n",
       "  'It was an incredibly happy customer service.']]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t = editor.template(['{it} {be} {a:pos_adj} {air_noun}.', '{it} {be} {a:intens} {pos_adj} {air_noun}.'] , intens=intens_adj, it=['It', 'This', 'That'], be=['is', 'was'], nsamples=500, save=True)\n",
    "t += editor.template(['{i} {pos_verb} {the} {air_noun}.', '{i} {intens} {pos_verb} {the} {air_noun}.'], intens=intens_verb, i=['I', 'We'], the=['this', 'that', 'the'], nsamples=500, save=True)\n",
    "t += editor.template(['{it} {be} {a:neg_adj} {air_noun}.', '{it} {be} {a:intens} {neg_adj} {air_noun}.'] , intens=intens_adj, it=['It', 'This', 'That'], be=['is', 'was'], nsamples=500, save=True)\n",
    "t += editor.template(['{i} {neg_verb} {the} {air_noun}.', '{i} {intens} {neg_verb} {the} {air_noun}.'], intens=intens_verb, i=['I', 'We'], the=['this', 'that', 'the'], nsamples=500, save=True)\n",
    "t.data[:5]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = editor.template(['{it} {be} {a:pos_adj} {air_noun}.', '{it} {be} {a:intens} {pos_adj} {air_noun}.'] , intens=intens_adj, it=['It', 'This', 'That'], be=['is', 'was'], nsamples=500, save=True)\n",
    "t += editor.template(['{i} {pos_verb} {the} {air_noun}.', '{i} {intens} {pos_verb} {the} {air_noun}.'], intens=intens_verb, i=['I', 'We'], the=['this', 'that', 'the'], nsamples=500, save=True)\n",
    "t += editor.template(['{it} {be} {a:neg_adj} {air_noun}.', '{it} {be} {a:intens} {neg_adj} {air_noun}.'] , intens=intens_adj, it=['It', 'This', 'That'], be=['is', 'was'], nsamples=500, save=True)\n",
    "t += editor.template(['{i} {neg_verb} {the} {air_noun}.', '{i} {intens} {neg_verb} {the} {air_noun}.'], intens=intens_verb, i=['I', 'We'], the=['this', 'that', 'the'], nsamples=500, save=True)\n",
    "test = DIR(t.data, monotonic_label, templates=t.templates)\n",
    "description = '''Test is composed of pairs of sentences (x1, x2), where we add an intensifier\n",
    "such as \"really\",or \"very\" to x2 and expect the confidence to NOT go down (with tolerance=0.1). e.g.:\n",
    "x1 = \"That was a good flight\"\n",
    "x2 = \"That was a very good flight\"\n",
    "We disregard cases where the prediction of x1 is neutral.\n",
    "'''\n",
    "suite.add(test, 'intensifiers', 'Vocabulary', description)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "reducer_adj = ['somewhat', 'kinda', 'mostly', 'probably', 'generally', 'reasonably', 'a little', 'a bit', 'slightly']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "monotonic_label_down = Expect.monotonic(increasing=False, tolerance=0.1)\n",
    "monotonic_label_down = Expect.slice_pairwise(monotonic_label_down, non_neutral_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['This customer service was sweet.',\n",
       "  'This customer service was kinda sweet.'],\n",
       " ['The customer service is nice.', 'The customer service is reasonably nice.'],\n",
       " ['This seat is adorable.', 'This seat is generally adorable.'],\n",
       " ['That customer service was fantastic.',\n",
       "  'That customer service was kinda fantastic.'],\n",
       " ['That food was wonderful.', 'That food was slightly wonderful.'],\n",
       " ['That aircraft was beautiful.', 'That aircraft was generally beautiful.'],\n",
       " ['That flight was sweet.', 'That flight was generally sweet.'],\n",
       " ['This aircraft is wonderful.', 'This aircraft is a little wonderful.'],\n",
       " ['The crew was perfect.', 'The crew was slightly perfect.'],\n",
       " ['The customer service is incredible.',\n",
       "  'The customer service is probably incredible.'],\n",
       " ['This customer service is nice.',\n",
       "  'This customer service is generally nice.'],\n",
       " ['This customer service is brilliant.',\n",
       "  'This customer service is kinda brilliant.'],\n",
       " ['That service is great.', 'That service is slightly great.'],\n",
       " ['The service was happy.', 'The service was probably happy.'],\n",
       " ['The aircraft was good.', 'The aircraft was generally good.'],\n",
       " ['The company was awesome.', 'The company was kinda awesome.'],\n",
       " ['That flight was excellent.', 'That flight was a little excellent.'],\n",
       " ['This company was brilliant.', 'This company was a bit brilliant.'],\n",
       " ['This pilot is fantastic.', 'This pilot is slightly fantastic.'],\n",
       " ['The plane was awesome.', 'The plane was slightly awesome.'],\n",
       " ['This staff was beautiful.', 'This staff was mostly beautiful.'],\n",
       " ['The company was great.', 'The company was a little great.'],\n",
       " ['That company was wonderful.', 'That company was reasonably wonderful.'],\n",
       " ['This seat is fun.', 'This seat is generally fun.'],\n",
       " ['This cabin crew was nice.', 'This cabin crew was a bit nice.'],\n",
       " ['This food is happy.', 'This food is kinda happy.'],\n",
       " ['The flight was fun.', 'The flight was a little fun.'],\n",
       " ['That company was fantastic.', 'That company was probably fantastic.'],\n",
       " ['This food is good.', 'This food is probably good.'],\n",
       " ['The cabin crew was beautiful.', 'The cabin crew was generally beautiful.'],\n",
       " ['This food is exceptional.', 'This food is probably exceptional.'],\n",
       " ['This airline is good.', 'This airline is somewhat good.'],\n",
       " ['That crew was sweet.', 'That crew was a bit sweet.'],\n",
       " ['This seat was exciting.', 'This seat was kinda exciting.'],\n",
       " ['That seat is amazing.', 'That seat is a little amazing.'],\n",
       " ['That cabin crew was great.', 'That cabin crew was generally great.'],\n",
       " ['This cabin crew was amazing.', 'This cabin crew was reasonably amazing.'],\n",
       " ['The service was nice.', 'The service was kinda nice.'],\n",
       " ['The seat is beautiful.', 'The seat is generally beautiful.'],\n",
       " ['This food is perfect.', 'This food is a little perfect.'],\n",
       " ['This plane was fun.', 'This plane was a bit fun.'],\n",
       " ['The customer service was brilliant.',\n",
       "  'The customer service was kinda brilliant.'],\n",
       " ['That staff was excellent.', 'That staff was a little excellent.'],\n",
       " ['The staff is nice.', 'The staff is a little nice.'],\n",
       " ['The staff was sweet.', 'The staff was kinda sweet.'],\n",
       " ['This airline was nice.', 'This airline was reasonably nice.'],\n",
       " ['The airline was good.', 'The airline was probably good.'],\n",
       " ['This service was adorable.', 'This service was mostly adorable.'],\n",
       " ['This flight was happy.', 'This flight was slightly happy.'],\n",
       " ['That cabin crew was fantastic.', 'That cabin crew was kinda fantastic.']]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "t = editor.template(['{it} {air_noun} {be} {pos_adj}.', '{it} {air_noun} {be} {red} {pos_adj}.'] , red=reducer_adj, it=['The', 'This', 'That'], be=['is', 'was'], nsamples=1000, save=True)\n",
    "t += editor.template(['{it} {air_noun} {be} {neg_adj}.', '{it} {air_noun} {be} {red} {neg_adj}.'] , red=reducer_adj, it=['The', 'This', 'That'], be=['is', 'was'], nsamples=1000, save=True)\n",
    "t.data[:50]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = editor.template(['{it} {air_noun} {be} {pos_adj}.', '{it} {air_noun} {be} {red} {pos_adj}.'] , red=reducer_adj, it=['The', 'This', 'That'], be=['is', 'was'], nsamples=1000, save=True)\n",
    "t += editor.template(['{it} {air_noun} {be} {neg_adj}.', '{it} {air_noun} {be} {red} {neg_adj}.'] , red=reducer_adj, it=['The', 'This', 'That'], be=['is', 'was'], nsamples=1000, save=True)\n",
    "test = DIR(t.data, monotonic_label_down, templates=t.templates)\n",
    "description = '''Test is composed of pairs of sentences (x1, x2), where we add a reducer\n",
    "such as \"somewhat\", or \"kinda\" to x2 and expect the confidence to NOT go up (with tolerance=0.1). e.g.:\n",
    "x1 = \"The cabin crew was good.\"\n",
    "x2 = \"The cabin crew was somewhat good.\"\n",
    "We disregard cases where the prediction of x1 is neutral.\n",
    "'''\n",
    "suite.add(test, 'reducers', 'Vocabulary', description)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### INVariance: change neutral words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "neutral_words = set(\n",
    "    ['.', 'the', 'The', ',', 'a', 'A', 'and', 'of', 'to', 'it', 'that', 'in',\n",
    "     'this', 'for',  'you', 'there', 'or', 'an', 'by', 'about', 'flight', 'my',\n",
    "     'in', 'of', 'have', 'with', 'was', 'at', 'it', 'get', 'from', 'this', 'Flight', 'plane'\n",
    "    ])\n",
    "forbidden = set(['No', 'no', 'Not', 'not', 'Nothing', 'nothing', 'without', 'but'] + pos_adj + neg_adj + pos_verb_present + pos_verb_past + neg_verb_present + neg_verb_past)\n",
    "def change_neutral(d):\n",
    "#     return d.text\n",
    "    examples = []\n",
    "    subs = []\n",
    "    words_in = [x for x in d.capitalize().split() if x in neutral_words]\n",
    "    if not words_in:\n",
    "        return None\n",
    "    for w in words_in:\n",
    "        suggestions = [x for x in editor.suggest_replace(d, w, beam_size=5, words_and_sentences=True) if x[0] not in forbidden]\n",
    "        examples.extend([x[1] for x in suggestions])\n",
    "        subs.extend(['%s -> %s' % (w, x[0]) for x in suggestions])\n",
    "    if examples:\n",
    "        idxs = np.random.choice(len(examples), min(len(examples), 10), replace=False)\n",
    "        return [examples[i] for i in idxs]#, [subs[i] for i in idxs])\n",
    "# Perturb.perturb(parsed_data[:5], perturb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = Perturb.perturb(sentences, change_neutral, nsamples=500)\n",
    "test = INV(t.data)\n",
    "description = 'Change a set of neutral words with other context-appropriate neutral words (using BERT).'\n",
    "suite.add(test, 'change neutral words with BERT', 'Vocabulary', description)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Add negative phrases"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "positive = editor.template('I {pos_verb_present} you.').data\n",
    "positive += editor.template('You are {pos_adj}.').data\n",
    "positive += ['I would fly with you again.']\n",
    "positive.remove('You are happy.')\n",
    "negative = editor.template('I {neg_verb_present} you.').data\n",
    "negative += editor.template('You are {neg_adj}.').data\n",
    "negative += ['Never flying with you again.']\n",
    "def add_phrase_function(phrases):\n",
    "    def pert(d):\n",
    "        while d[-1].pos_ == 'PUNCT':\n",
    "            d = d[:-1]\n",
    "        d = d.text\n",
    "        ret = [d + '. ' + x for x in phrases]\n",
    "        idx = np.random.choice(len(ret), 10, replace=False)\n",
    "        ret = [ret[i] for i in idx]\n",
    "        return ret\n",
    "    return pert\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "def positive_change(orig_conf, conf):\n",
    "    softmax = type(orig_conf) in [np.array, np.ndarray]\n",
    "    if not softmax or orig_conf.shape[0] != 3:\n",
    "        raise(Exception('Need prediction function to be softmax with 3 labels (negative, neutral, positive)'))\n",
    "    return orig_conf[0] - conf[0] + conf[2] - orig_conf[2]\n",
    "\n",
    "def diff_up(orig_pred, pred, orig_conf, conf, labels=None, meta=None):\n",
    "    tolerance = 0.1\n",
    "    change = positive_change(orig_conf, conf)\n",
    "    if change + tolerance >= 0:\n",
    "        return True\n",
    "    else:\n",
    "        return change + tolerance\n",
    "def diff_down(orig_pred, pred, orig_conf, conf, labels=None, meta=None):\n",
    "    tolerance = 0.1\n",
    "    change = positive_change(orig_conf, conf)\n",
    "    if change - tolerance <= 0:\n",
    "        return True\n",
    "    else:\n",
    "        return -(change - tolerance)\n",
    "goes_up = Expect.pairwise(diff_up)\n",
    "goes_down = Expect.pairwise(diff_down)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = Perturb.perturb(parsed_data, add_phrase_function(positive), nsamples=500)\n",
    "test = DIR(t.data, goes_up)\n",
    "description = 'Add very positive phrases (e.g. I love you) to the end of sentences, expect probability of positive to NOT go down (tolerance=0.1)'\n",
    "suite.add(test, 'add positive phrases', 'Vocabulary', description)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = Perturb.perturb(parsed_data, add_phrase_function(negative), nsamples=500)\n",
    "test = DIR(t.data, goes_down)\n",
    "description = 'Add very negative phrases (e.g. I hate you) to the end of sentences, expect probability of positive to NOT go up (tolerance=0.1)'\n",
    "suite.add(test, 'add negative phrases', 'Vocabulary', description)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Capability: robustness\n",
    "### INVariance: adding irrelevant stuff before and after.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "import string\n",
    "def random_string(n):\n",
    "    return ''.join(np.random.choice([x for x in string.ascii_letters + string.digits], n))\n",
    "def random_url(n=6):\n",
    "    return 'https://t.co/%s' % random_string(n)\n",
    "def random_handle(n=6):\n",
    "    return '@%s' % random_string(n)\n",
    "\n",
    "# data['sentence']\n",
    "\n",
    "def add_irrelevant(sentence):\n",
    "    urls_and_handles = [random_url(n=6) for _ in range(5)] + [random_handle() for _ in range(5)]\n",
    "    irrelevant_before = ['@airline '] + urls_and_handles\n",
    "    irrelevant_after = urls_and_handles \n",
    "    rets = ['%s %s' % (x, sentence) for x in irrelevant_before ]\n",
    "    rets += ['%s %s' % (sentence, x) for x in irrelevant_after]\n",
    "    return rets\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = Perturb.perturb(sentences, add_irrelevant, nsamples=500)\n",
    "test = INV(t.data)\n",
    "suite.add(test, 'add random urls and handles', 'Robustness', 'add randomly generated urls and handles to the start or end of sentence')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### punctuation, contractions, typos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = Perturb.perturb(parsed_data, Perturb.punctuation, nsamples=500)\n",
    "test = INV(t.data)\n",
    "suite.add(test, 'punctuation', 'Robustness', 'strip punctuation and / or add \".\"')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = Perturb.perturb(sentences, Perturb.add_typos, nsamples=500, typos=1)\n",
    "test = INV(t.data)\n",
    "suite.add(test, 'typos', 'Robustness', 'Add one typo to input by swapping two adjacent characters')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = Perturb.perturb(sentences, Perturb.add_typos, nsamples=500, typos=2)\n",
    "test = INV(t.data)\n",
    "suite.add(test, '2 typos', 'Robustness', 'Add two typos to input by swapping two adjacent characters twice')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = Perturb.perturb(sentences, Perturb.contractions, nsamples=1000)\n",
    "test = INV(t.data)\n",
    "suite.add(test, 'contractions', 'Robustness', 'Contract or expand contractions, e.g. What is -> What\\'s')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Capability: NER"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = Perturb.perturb(parsed_data, Perturb.change_names, nsamples=1000)\n",
    "test = INV(t.data)\n",
    "suite.add(test, 'change names', 'NER', 'Replace names with other common names')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = Perturb.perturb(parsed_data, Perturb.change_location, nsamples=1000)\n",
    "test = INV(t.data)\n",
    "suite.add(test, 'change locations', 'NER', 'Replace city or country names with other cities or countries')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = Perturb.perturb(parsed_data, Perturb.change_number, nsamples=1000)\n",
    "test = INV(t.data)\n",
    "suite.add(test, 'change numbers', 'NER', 'Replace integers with random integers within a 20% radius of the original')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Capability: temporal awareness"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['hate', 'dislike', 'regret', 'abhor', 'dread', 'despise']"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "editor.template('{neg_verb_present}').data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "change = ['but', 'even though', 'although', '']\n",
    "t = editor.template(['I used to think this airline was {neg_adj}, {change} now I think it is {pos_adj}.',\n",
    "                                 'I think this airline is {pos_adj}, {change} I used to think it was {neg_adj}.',\n",
    "                                 'In the past I thought this airline was {neg_adj}, {change} now I think it is {pos_adj}.',\n",
    "                                 'I think this airline is {pos_adj}, {change} in the past I thought it was {neg_adj}.',\n",
    "                                ] ,\n",
    "                                 change=change, unroll=True, nsamples=500, save=True, labels=2)\n",
    "t += editor.template(['I used to {neg_verb_present} this airline, {change} now I {pos_verb_present} it.',\n",
    "                                 'I {pos_verb_present} this airline, {change} I used to {neg_verb_present} it.',\n",
    "                                 'In the past I would {neg_verb_present} this airline, {change} now I {pos_verb} it.',\n",
    "                                 'I {pos_verb_present} this airline, {change} in the past I would {neg_verb_present} it.',\n",
    "                                ] ,\n",
    "                                change=change, unroll=True, nsamples=500, save=True, labels=2)\n",
    "\n",
    "t += editor.template(['I used to think this airline was {pos_adj}, {change} now I think it is {neg_adj}.',\n",
    "                                 'I think this airline is {neg_adj}, {change} I used to think it was {pos_adj}.',\n",
    "                                 'In the past I thought this airline was {pos_adj}, {change} now I think it is {neg_adj}.',\n",
    "                                 'I think this airline is {neg_adj}, {change} in the past I thought it was {pos_adj}.',\n",
    "                                ] ,\n",
    "                                 change=change, unroll=True, nsamples=500, save=True, labels=0)\n",
    "t += editor.template(['I used to {pos_verb_present} this airline, {change} now I {neg_verb_present} it.',\n",
    "                                 'I {neg_verb_present} this airline, {change} I used to {pos_verb_present} it.',\n",
    "                                 'In the past I would {pos_verb_present} this airline, {change} now I {neg_verb_present} it.',\n",
    "                                 'I {neg_verb_present} this airline, {change} in the past I would {pos_verb_present} it.',\n",
    "                                ] ,\n",
    "                                change=change, unroll=True, nsamples=500, save=True, labels=0)\n",
    "test = MFT(**t)\n",
    "description = '''Have two conflicing statements, one about the past and one about the present.\n",
    "Expect the present to carry the sentiment. Examples:\n",
    "I used to love this airline, now I hate it -> should be negative\n",
    "I love this airline, although I used to hate it -> should be positive\n",
    "'''\n",
    "suite.add(test, 'used to, but now', 'Temporal', description)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "used to should reduce"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['it is a good flight.', 'I used to think it is a good flight.'],\n",
       " ['this is a good flight.', 'I used to think this is a good flight.'],\n",
       " ['that is a good flight.', 'I used to think that is a good flight.'],\n",
       " ['it was a good flight.', 'I used to think it was a good flight.'],\n",
       " ['this was a good flight.', 'I used to think this was a good flight.']]"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t = editor.template(['{it} {be} {a:adj} {air_noun}.', 'I used to think {it} {be} {a:adj} {air_noun}.'], it=['it', 'this', 'that'], be=['is', 'was'], adj=editor.lexicons['pos_adj'] + editor.lexicons['neg_adj'], save=True)\n",
    "t += editor.template(['{i} {verb} {the} {air_noun}.', '{i} used to {verb} {the} {air_noun}.'], i=['I', 'We'], the=['this', 'that', 'the'], verb=editor.lexicons['pos_verb_present'] + editor.lexicons['neg_verb_present'], save=True)\n",
    "t.data[:5]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = editor.template(['{it} {be} {a:adj} {air_noun}.', 'I used to think {it} {be} {a:adj} {air_noun}.'], it=['it', 'this', 'that'], be=['is', 'was'], adj=editor.lexicons['pos_adj'] + editor.lexicons['neg_adj'], save=True)\n",
    "t += editor.template(['{i} {verb} {the} {air_noun}.', '{i} used to {verb} {the} {air_noun}.'], i=['I', 'We'], the=['this', 'that', 'the'], verb=editor.lexicons['pos_verb_present'] + editor.lexicons['neg_verb_present'], save=True)\n",
    "test = DIR(t.data, monotonic_label_down, templates=t.templates)\n",
    "suite.add(test, '\"used to\" should reduce', 'Temporal', 'A model should not be more confident on \"I used to think X\" when compared to \"X\", e.g. \"I used to love this airline\" should have less confidence than \"I love this airline\"')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Capability: Fairness"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['male', 'female', 'first_name', 'first_pronoun', 'last_name', 'country', 'nationality', 'city', 'religion', 'religion_adj', 'sexual_adj', 'air_noun', 'pos_adj', 'neg_adj', 'neutral_adj', 'pos_verb_present', 'neg_verb_present', 'neutral_verb_present', 'pos_verb_past', 'neg_verb_past', 'neutral_verb_past', 'pos_verb', 'neg_verb', 'neutral_verb'])"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "editor.lexicons.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['a Christian',\n",
       " 'a Protestant',\n",
       " 'a Roman Catholic',\n",
       " 'an Eastern Orthodox',\n",
       " 'an Anglican',\n",
       " 'a Jew',\n",
       " 'an Orthodox Jew',\n",
       " 'a Muslim',\n",
       " 'a Sunni',\n",
       " \"a Shi'a\",\n",
       " 'an Ahmadiyya',\n",
       " 'a Buddhist',\n",
       " 'a Zoroastrian',\n",
       " 'a Hindu',\n",
       " 'a Sikh',\n",
       " 'a Shinto',\n",
       " \"a Baha'i\",\n",
       " 'a Taoist',\n",
       " 'a Confucian',\n",
       " 'a Jain',\n",
       " 'an Atheist',\n",
       " 'an Agnostic']"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "editor.template('{a:religion_adj}').data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "protected = {\n",
    "    'race': ['a black','a hispanic', 'a white', 'an asian'],\n",
    "    'sexual': editor.template('{a:sexual_adj}').data,\n",
    "    'religion': editor.template('{a:religion_adj}').data,\n",
    "    'nationality': editor.template('{a:nationality}').data[:20],\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "# t.data[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "race\n",
      "sexual\n",
      "religion\n",
      "nationality\n"
     ]
    }
   ],
   "source": [
    "for p, vals in protected.items():\n",
    "    print(p)\n",
    "    t = editor.template(['{male} is %s {mask}.' % r for r in vals], return_maps=False, nsamples=300, save=True)\n",
    "    t += editor.template(['{female} is %s {mask}.' % r for r in vals], return_maps=False, nsamples=300, save=True)\n",
    "    test = INV(t.data, threshold=0.1, templates=t.templates)\n",
    "    suite.add(test, 'protected: %s' % p, 'Fairness', 'Prediction should be the same for various adjectives within a protected class')\n",
    "#     test.run(new_pp)\n",
    "#     test.summary(n=3)\n",
    "#     print()\n",
    "#     preds = np.array(test.results.preds)\n",
    "#     for i, x in enumerate(vals):\n",
    "#         print('%.2f %s' % (preds[:, i].mean(), vals[i]))\n",
    "#     print()\n",
    "#     print()\n",
    "#     print('-------------------------')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "## Capability: Negation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Simple templates:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = editor.template('{it} {air_noun} {nt} {pos_adj}.', it=['This', 'That', 'The'], nt=['is not', 'isn\\'t'], save=True)\n",
    "t += editor.template('{it} {benot} {a:pos_adj} {air_noun}.', it=['It', 'This', 'That'], benot=['is not',  'isn\\'t', 'was not', 'wasn\\'t'], save=True)\n",
    "neg = ['I can\\'t say I', 'I don\\'t', 'I would never say I', 'I don\\'t think I', 'I didn\\'t' ]\n",
    "t += editor.template('{neg} {pos_verb_present} {the} {air_noun}.', neg=neg, the=['this', 'that', 'the'], save=True)\n",
    "t += editor.template('No one {pos_verb_present}s {the} {air_noun}.', neg=neg, the=['this', 'that', 'the'], save=True)\n",
    "test = MFT(t.data, labels=0, templates=t.templates)\n",
    "suite.add(test, 'simple negations: negative', 'Negation', 'Very simple negations of positive statements')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = editor.template('{it} {air_noun} {nt} {neg_adj}.', it=['This', 'That', 'The'], nt=['is not', 'isn\\'t'], save=True)\n",
    "t += editor.template('{it} {benot} {a:neg_adj} {air_noun}.', it=['It', 'This', 'That'], benot=['is not',  'isn\\'t', 'was not', 'wasn\\'t'], save=True)\n",
    "neg = ['I can\\'t say I', 'I don\\'t', 'I would never say I', 'I don\\'t think I', 'I didn\\'t' ]\n",
    "t += editor.template('{neg} {neg_verb_present} {the} {air_noun}.', neg=neg, the=['this', 'that', 'the'], save=True)\n",
    "t += editor.template('No one {neg_verb_present}s {the} {air_noun}.', neg=neg, the=['this', 'that', 'the'], save=True)\n",
    "# expectation: prediction is not 0\n",
    "is_not_0 = lambda x, pred, *args: pred != 0\n",
    "test = MFT(t.data, Expect.single(is_not_0), templates=t.templates)\n",
    "suite.add(test, 'simple negations: not negative', 'Negation', 'Very simple negations of negative statements. Expectation requires prediction to NOT be negative (i.e. neutral or positive)')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = editor.template('{it} {air_noun} {nt} {neutral_adj}.', it=['This', 'That', 'The'], nt=['is not', 'isn\\'t'], save=True)\n",
    "t += editor.template('{it} {benot} {a:neutral_adj} {air_noun}.', it=['It', 'This', 'That'], benot=['is not',  'isn\\'t', 'was not', 'wasn\\'t'], save=True)\n",
    "neg = ['I can\\'t say I', 'I don\\'t', 'I would never say I', 'I don\\'t think I', 'I didn\\'t' ]\n",
    "t += editor.template('{neg} {neutral_verb_present} {the} {air_noun}.', neg=neg, the=['this', 'that', 'the'], save=True)\n",
    "test = MFT(t.data, labels=1, templates=t.templates)\n",
    "suite.add(test, 'simple negations: not neutral is still neutral', 'Negation', 'Negating neutral statements should still result in neutral predictions')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Different templates:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "air_noun_it = [x for x in editor.lexicons['air_noun'] if x != 'pilot']\n",
    "t = editor.template('I thought {it} {air_noun} would be {pos_adj}, but it {neg}.', air_noun=air_noun_it, neg=['was not', 'wasn\\'t'], it=['this', 'that', 'the'], nt=['is not', 'isn\\'t'], save=True)\n",
    "t += editor.template('I thought I would {pos_verb_present} {the} {air_noun}, but I {neg}.', neg=['did not', 'didn\\'t'], the=['this', 'that', 'the'], save=True)\n",
    "test = MFT(t.data, labels=0, templates=t.templates)\n",
    "suite.add(test, 'simple negations: I thought x was positive, but it was not (should be negative)', 'Negation', '', overwrite=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = editor.template('I thought {it} {air_noun} would be {neg_adj}, but it {neg}.', air_noun=air_noun_it, neg=['was not', 'wasn\\'t'], it=['this', 'that', 'the'], nt=['is not', 'isn\\'t'], save=True)\n",
    "t += editor.template('I thought I would {neg_verb_present} {the} {air_noun}, but I {neg}.', neg=['did not', 'didn\\'t'], the=['this', 'that', 'the'], save=True)\n",
    "# expectation: prediction is not 0\n",
    "test = MFT(t.data, Expect.single(is_not_0), templates=t.templates)\n",
    "suite.add(test, 'simple negations: I thought x was negative, but it was not (should be neutral or positive)', 'Negation', '')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = editor.template('I thought {it} {air_noun} would be {neutral_adj}, but it {neg}.', air_noun=air_noun_it, neg=['was not', 'wasn\\'t'], it=['this', 'that', 'the'], nt=['is not', 'isn\\'t'], save=True)\n",
    "t += editor.template('I thought I would {neutral_verb_present} {the} {air_noun}, but I {neg}.', neg=['did not', 'didn\\'t'], the=['this', 'that', 'the'], save=True)\n",
    "# expectation: prediction is not 0\n",
    "test = MFT(t.data, labels=1, templates=t.templates)\n",
    "suite.add(test, 'simple negations: but it was not (neutral) should still be neutral', 'Negation', '')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Harder: negation with neutral in the middle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_neg = neg[:-1]\n",
    "neutral =['that I am from Brazil', 'my history with airplanes', 'all that I\\'ve seen over the years', 'the time that I\\'ve been flying', 'it\\'s a Tuesday']\n",
    "t = editor.template('{neg}, given {neutral}, that {it} {air_noun} {be} {pos_adj}.', neutral=neutral, neg=['I don\\'t think', 'I can\\'t say', 'I wouldn\\'t say'], it=['this', 'that', 'the'], be=['is', 'was'], save=True)\n",
    "t += editor.template('{neg}, given {neutral}, that {it} {be} {a:pos_adj} {air_noun}.',neutral=neutral,  neg=['I don\\'t think', 'I can\\'t say', 'I wouldn\\'t say'], it=['this', 'that', 'the'], be=['is', 'was'], save=True)\n",
    "t += editor.template('{neg}, given {neutral}, that {i} {pos_verb_present} {the} {air_noun}.',neutral=neutral,  neg=['I don\\'t think', 'I can\\'t say', 'I wouldn\\'t say'], i=['I', 'we'], the=['this', 'that', 'the'], save=True)\n",
    "t.data = list(np.random.choice(t.data, 1000, replace=False))\n",
    "test = MFT(t.data, labels=0, templates=t.templates)\n",
    "suite.add(test, 'Hard: Negation of positive with neutral stuff in the middle (should be negative)', 'Negation', '')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "neutral =['that I am from Brazil', 'my history with airplanes', 'all that I\\'ve seen over the years', 'the time that I\\'ve been flying', 'it\\'s a Tuesday']\n",
    "t = editor.template('{neg}, given {neutral}, that {it} {air_noun} {be} {neg_adj}.', neutral=neutral, neg=['I don\\'t think', 'I can\\'t say', 'I wouldn\\'t say'], it=['this', 'that', 'the'], be=['is', 'was'], save=True)\n",
    "t += editor.template('{neg}, given {neutral}, that {it} {be} {a:neg_adj} {air_noun}.',neutral=neutral,  neg=['i don\\'t think', 'i can\\'t say', 'i wouldn\\'t say'], it=['this', 'that', 'the'], be=['is', 'was'], save=True)\n",
    "t += editor.template('{neg}, given {neutral}, that {i} {neg_verb_present} {the} {air_noun}.',neutral=neutral,  neg=['i don\\'t think', 'i can\\'t say', 'i wouldn\\'t say'], i=['I', 'we'], the=['this', 'that', 'the'], save=True)\n",
    "t.data = list(np.random.choice(t.data, 1000, replace=False))\n",
    "test = MFT(t.data, Expect.single(is_not_0), templates=t.templates)\n",
    "suite.add(test, 'Hard: Negation of negative with neutral stuff in the middle (should be positive or neutral)', 'Negation', '')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "neutral =['that I am from Brazil', 'my history with airplanes', 'all that I\\'ve seen over the years', 'the time that I\\'ve been flying', 'it\\'s a Tuesday']\n",
    "t = editor.template('{neg}, given {neutral}, that {it} {air_noun} {be} {neutral_adj}.', neutral=neutral, neg=['I don\\'t think', 'I can\\'t say', 'I wouldn\\'t say'], it=['this', 'that', 'the'], be=['is', 'was'], save=True)\n",
    "t += editor.template('{neg}, given {neutral}, that {it} {be} {a:neutral_adj} {air_noun}.',neutral=neutral,  neg=['I don\\'t think', 'I can\\'t say', 'I wouldn\\'t say'], it=['this', 'that', 'the'], be=['is', 'was'], save=True)\n",
    "t += editor.template('{neg}, given {neutral}, that {i} {neutral_verb_present} {the} {air_noun}.',neutral=neutral,  neg=['I don\\'t think', 'I can\\'t say', 'I wouldn\\'t say'], i=['I', 'we'], the=['this', 'that', 'the'], save=True)\n",
    "t.data = list(np.random.choice(t.data, 1000, replace=False))\n",
    "test = MFT(t.data, labels=1, templates=t.templates)\n",
    "suite.add(test, 'negation of neutral with neutral in the middle, should still neutral', 'Negation', '')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "## Capability: SRL"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "my opinion is more important than others"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "change = [' but', '']\n",
    "templates = ['Some people think you are {neg_adj},{change} I think you are {pos_adj}.',\n",
    "             'I think you are {pos_adj},{change} some people think you are {neg_adj}.',\n",
    "             'I had heard you were {neg_adj},{change} I think you are {pos_adj}.',\n",
    "             'I think you are {pos_adj},{change} I had heard you were {neg_adj}.',\n",
    "             ]\n",
    "t = editor.template(templates, change=change, unroll=True, labels=2, save=True)\n",
    "templates = ['{others} {neg_verb_present} you,{change} I {pos_verb_present} you.',\n",
    "             'I {pos_verb_present} you,{change} {others} {neg_verb_present} you.',\n",
    "            ]\n",
    "others = ['some people', 'my parents', 'my friends', 'people']\n",
    "t += editor.template(templates, others=others, change=change, unroll=True, labels=2, save=True)\n",
    "\n",
    "change = [' but', '']\n",
    "templates = ['Some people think you are {pos_adj},{change} I think you are {neg_adj}.',\n",
    "             'I think you are {neg_adj},{change} some people think you are {pos_adj}.',\n",
    "             'I had heard you were {pos_adj},{change} I think you are {neg_adj}.',\n",
    "             'I think you are {neg_adj},{change} I had heard you were {pos_adj}.',\n",
    "             ]\n",
    "t += editor.template(templates, change=change, unroll=True, labels=0, save=True)\n",
    "templates = ['{others} {pos_verb_present} you,{change} I {neg_verb_present} you.',\n",
    "             'I {neg_verb_present} you,{change} {others} {pos_verb_present} you.',\n",
    "            ]\n",
    "others = ['some people', 'my parents', 'my friends', 'people']\n",
    "t += editor.template(templates, others=others, change=change, unroll=True, labels=0, save=True)\n",
    "test = MFT(**t)\n",
    "description = '''Have conflicting statements where the author has an opinion and a third party has a contrary opinion.\n",
    "Expect sentiment to be the authors'. Example:\n",
    "\"Some people think you are great, but I think you are terrible\" -> should be negative\n",
    "'''\n",
    "suite.add(test, 'my opinion is what matters', 'SRL', description)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "q & a form: yes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = editor.template('Do I think {it} {air_noun} {be} {pos_adj}? Yes', it=['that', 'this', 'the'], be=['is', 'was'], save=True, labels=2)\n",
    "t += editor.template('Do I think {it} {be} {a:pos_adj} {air_noun}? Yes', it=['it', 'this', 'that'], be=['is', 'was'], save=True, labels=2)\n",
    "t += editor.template('Did {i} {pos_verb_present} {the} {air_noun}? Yes', i=['I', 'we'], the=['this', 'that', 'the'], save=True, labels=2)\n",
    "t += editor.template('Do I think {it} {air_noun} {be} {neg_adj}? Yes', it=['that', 'this', 'the'], be=['is', 'was'], save=True, labels=0)\n",
    "t += editor.template('Do I think {it} {be} {a:neg_adj} {air_noun}? Yes', it=['it', 'this', 'that'], be=['is', 'was'], save=True, labels=0)\n",
    "t += editor.template('Did {i} {neg_verb_present} {the} {air_noun}? Yes', i=['I', 'we'], the=['this', 'that', 'the'], save=True, labels=0)\n",
    "test = MFT(**t)\n",
    "suite.add(test, 'Q & A: yes', 'SRL', 'TODO_DESCRIPTION')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = editor.template('Do I think {it} {air_noun} {be} {neutral_adj}? Yes', it=['that', 'this', 'the'], be=['is', 'was'], save=True)\n",
    "t += editor.template('Do I think {it} {be} {a:neutral_adj} {air_noun}? Yes', it=['it', 'this', 'that'], be=['is', 'was'], save=True)\n",
    "t += editor.template('Did {i} {neutral_verb_present} {the} {air_noun}? Yes', i=['I', 'we'], the=['this', 'that', 'the'], save=True)\n",
    "test = MFT(t.data, labels=1, templates=t.templates)\n",
    "suite.add(test, 'Q & A: yes (neutral)', 'SRL', 'TODO_DESCRIPTION')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = editor.template('Do I think {it} {air_noun} {be} {pos_adj}? No', it=['that', 'this', 'the'], be=['is', 'was'], save=True, labels=0)\n",
    "t += editor.template('Do I think {it} {be} {a:pos_adj} {air_noun}? No', it=['it', 'this', 'that'], be=['is', 'was'], save=True, labels=0)\n",
    "t += editor.template('Did {i} {pos_verb_present} {the} {air_noun}? No', i=['I', 'we'], the=['this', 'that', 'the'], save=True, labels=0)\n",
    "t += editor.template('Do I think {it} {air_noun} {be} {neg_adj}? No', it=['that', 'this', 'the'], be=['is', 'was'], save=True, labels=1)\n",
    "t += editor.template('Do I think {it} {be} {a:neg_adj} {air_noun}? No', it=['it', 'this', 'that'], be=['is', 'was'], save=True, labels=1)\n",
    "t += editor.template('Did {i} {neg_verb_present} {the} {air_noun}? No', i=['I', 'we'], the=['this', 'that', 'the'], save=True, labels=1)\n",
    "allow_for_neutral = lambda x, pred, _, label, _2 : pred != 0 if label == 1 else pred == label\n",
    "test = MFT(t.data, Expect.single(allow_for_neutral), labels=t.labels, templates=t.templates)\n",
    "suite.add(test, 'Q & A: no', 'SRL', 'TODO_DESCRIPTION')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = editor.template('Do I think {it} {air_noun} {be} {neutral_adj}? No', it=['that', 'this', 'the'], be=['is', 'was'], save=True)\n",
    "t += editor.template('Do I think {it} {be} {a:neutral_adj} {air_noun}? No', it=['it', 'this', 'that'], be=['is', 'was'], save=True)\n",
    "t += editor.template('Did {i} {neutral_verb_present} {the} {air_noun}? No', i=['I', 'we'], the=['this', 'that', 'the'], save=True)\n",
    "test = MFT(t.data, labels=1, templates=t.templates)\n",
    "suite.add(test, 'Q & A: no (neutral)', 'SRL', 'TODO_DESCRIPTION')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "suite.to_raw_file('/tmp/temp2', n=500, seed=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "for test in suite.tests:\n",
    "    suite.tests[test].name = test\n",
    "    suite.tests[test].description = suite.info[test]['description]']\n",
    "    suite.tests[test].capability = suite.info[test]['capability']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = '/home/marcotcr/work/checklist/release_data/sentiment/sentiment_suite.pkl'\n",
    "suite.save(path)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "checklist",
   "language": "python",
   "name": "checklist"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
